# -*- coding: utf-8 -*-
import scrapy
from copy import deepcopy
from scrapy_redis.spiders import RedisSpider
from Books.items import DdItem
import urllib

class JdSpider(RedisSpider):
    name = 'dd'
    allowed_domains = ['dangdang.com']
    # start_urls = ['http://book.dangdang.com/']
    redis_key = 'dd:start_urls'

    def parse(self, response):
        # 大分类分组
        div_list = response.xpath('//div[@class="con flq_body"]/div')
        for div in div_list:
            item = DdItem()
            item["b_cate"] = div.xpath('./dl/dt//text()').getall()
            # 中分类分组
            dl_list = div.xpath('./div//dl[contains(@class,"inner_dl")]')
            for dl in dl_list:
                item["m_cate"] = dl.xpath('./dt//text()').getall()
                # 小分类分组
                a_list = dl.xpath('./dd/a')
                for a in a_list:
                    item["s_href"] = a.xpath('./@href').get()
                    item["s_cate"] = a.xpath('./@title').get()
                    if item["s_href"] is not None:
                        yield scrapy.Request(
                            url=item["s_href"],
                            callback=self.parse_book_list,
                            meta={"item":deepcopy(item)}
                        )

    def parse_book_list(self, response):
        item = response.meta["item"]
        # 书单分组
        li_list = response.xpath('.//ul[@class="bigimg"]/li')
        for li in li_list:
            item["book_img"] = li.xpath('./a[@class="pic"]/img/@src').get()
            if 'url_none' in item["book_img"]:
                item["book_img"] = li.xpath('./a[@class="pic"]/img/@data-original').get()
            item["book_name"] = li.xpath('./p[@class="name"]/a/@title').get()
            item["book_desc"] = li.xpath('./p[@class="detail"]/text()').get()
            item["book_price"] = li.xpath('.//span[@class="search_now_price"]/text()').get()
            item['book_author'] = li.xpath('./p[@class="search_book_author"]/span[1]/a/text()').get()
            item['book_publish_date'] = li.xpath('./p[@class="search_book_author"]/span[2]/text()').get()
            item['book_press'] = li.xpath('./p[@class="search_book_author"]/span[3]/a/@title').get()
            yield item
        # 翻页
        next_url = response.xpath('//li[@class="next"]/a/@href').get()
        if next_url is not None:
            next_url = urllib.parse.urljoin(response.url, next_url)
            yield  scrapy.Request(
                url=next_url,
                callback=self.parse_book_list,
                meta={"item":item}
            )